package pri.lt;

import com.gargoylesoftware.htmlunit.*;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.javascript.JavaScriptErrorListener;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.http.*;
import org.springframework.http.HttpMethod;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
import org.thunlp.text.classifiers.BasicTextClassifier;
import org.thunlp.text.classifiers.ClassifyResult;
import org.thunlp.text.classifiers.LinearBigramChineseTextClassifier;
import pri.lt.bean.BBSResponse;
import com.quark.common.entity.Posts;
import pri.lt.client.LabelClient;
import pri.lt.entity.FetchHistory;
import pri.lt.parser.Parser;
import pri.lt.parser.ParserManager;

import java.io.File;
import java.io.FileOutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Pattern;

public class MyCrawler extends WebCrawler {

    private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg"
            + "|png|mp3|mp4|zip|gz))$");

    private static String userToken = null;


    /**
     * This method receives two parameters. The first parameter is the page
     * in which we have discovered this new url and the second parameter is
     * the new url. You should implement this function to specify whether
     * the given url should be crawled or not (based on your crawling logic).
     * In this example, we are instructing the crawler to ignore urls that
     * have css, js, git, ... extensions and to only accept urls that start
     * with "https://www.ics.uci.edu/". In this case, we didn't need the
     * referringPage parameter to make the decision.
     */
    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();

        boolean result = false;

        boolean notFilter = !FILTERS.matcher(href).matches();

        boolean supportLink = notFilter
                && ParserManager.isDomainSupport(href);

        boolean newsPage = notFilter
                && ParserManager.isUrlCanParse(href);

        boolean domainDirectory = notFilter
                && ParserManager.isDomainDirectory(href);

        if (supportLink) {
            if (newsPage) {
                try {
                    List<FetchHistory> fetchHistories = App.fetchHistoryDao.findByUrl(href);
                    if (fetchHistories == null || fetchHistories.size() == 0) {
                        result = true;
                    } else {
                        if (!fetchHistories.get(0).getGood()) {
                            Parser.ParserResult parserResult = null;
                            parserResult = new Parser.ParserResult();
                            parserResult.publishTime = fetchHistories.get(0).getInitTime();
                            parserResult.thumbnailUrl = fetchHistories.get(0).getThumbUrl();
                            parserResult.title = fetchHistories.get(0).getTitle();
                            parserResult.content = fetchHistories.get(0).getContent();
                            //parserResult.allHtml = fetchHistories.get(0).getAllHtml();
                            parserResult.commentUrl = fetchHistories.get(0).getCommentUrl();
                            Parser.HotCommentResult hotCommentResult = ParserManager.getHotCommentResult(href, parserResult);
                            if (hotCommentResult != null && hotCommentResult.isHot) {
                                publishPost(fetchHistories.get(0), hotCommentResult, href);
                            }
                        }
                    }
                } catch (Exception e) {

                    e.printStackTrace();
                }
            } else {
                if (domainDirectory) {
                    result = true;
                }
            }
        }

        return result;
    }

    /**
     * This function is called when a page is fetched and ready
     * to be processed by your program.
     */
    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        System.out.println("URL: " + url);
        if(ParserManager.isUrlCanParse(url)) {
            if (page.getParseData() instanceof HtmlParseData) {
                HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
                String text = htmlParseData.getText();
                String html = htmlParseData.getHtml();
                Set<WebURL> links = htmlParseData.getOutgoingUrls();

                System.out.println("Text length: " + text.length());
                System.out.println("Html length: " + html.length());
                System.out.println("Number of outgoing links: " + links.size());


                Parser.ParserResult parserResult = null;
                FetchHistory savedFetchHistory = null;
                List<FetchHistory> fetchHistories = App.fetchHistoryDao.findByUrl(url);
                if (fetchHistories != null && fetchHistories.size() > 0) {
                    savedFetchHistory = fetchHistories.get(0);
                    if (!savedFetchHistory.getGood()) {
                        parserResult = new Parser.ParserResult();
                        parserResult.publishTime = savedFetchHistory.getInitTime();
                        parserResult.thumbnailUrl = savedFetchHistory.getThumbUrl();
                        parserResult.title = savedFetchHistory.getTitle();
                        parserResult.content = savedFetchHistory.getContent();
                        //parserResult.allHtml = savedFetchHistory.getAllHtml();
                        parserResult.commentUrl = savedFetchHistory.getCommentUrl();
                    }
                } else {
                    parserResult = ParserManager.parser(url);
                    if (parserResult != null) {
                        FetchHistory fetchHistory = new FetchHistory();
                        fetchHistory.setContent(parserResult.content);
                        fetchHistory.setInitTime(parserResult.publishTime);
                        fetchHistory.setOriginUrl(url);
                        fetchHistory.setTitle(parserResult.title);
                        fetchHistory.setGood(false);
                        fetchHistory.setThumbUrl(parserResult.thumbnailUrl);
                        //fetchHistory.setAllHtml(parserResult.allHtml);
                        fetchHistory.setCommentUrl(parserResult.commentUrl);
                        savedFetchHistory = App.fetchHistoryDao.save(fetchHistory);
                    }
                }

                try {
                    if (parserResult != null) {
//                    File file = new File("result/" + parserResult.title + ".html");
//                    String path = file.getAbsolutePath();
//                    System.out.println("path " + path);
//                    //System.out.println("Html : " + html);
//                    if (!file.exists()) {
//                        file.createNewFile();
//                    }
//                    FileOutputStream outputStream = new FileOutputStream(file);
//                        outputStream.write(parserResult.allHtml.getBytes());
//                    outputStream.close();


                        Parser.HotCommentResult hotCommentResult = ParserManager.getHotCommentResult(url, parserResult);
                        if (hotCommentResult != null && hotCommentResult.isHot) {
                            publishPost(savedFetchHistory, hotCommentResult, url);
                        }


                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }



    private static void updateBBSUserToken() {
        if (userToken == null || userToken.trim().length() == 0) {
            RestTemplate restTemplate = new RestTemplate();

            HttpHeaders headers = new HttpHeaders();
//  //请勿轻易改变此提交方式，大部分的情况下，提交方式都是表单提交
            headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
//  //封装参数，千万不要替换为Map与HashMap，否则参数无法传递
            MultiValueMap<String, String> params2= new LinkedMultiValueMap<String, String>();
//  //也支持中文
            params2.add("email", "1420432344@qq.com");
            params2.add("password", "12345678");
            HttpEntity<MultiValueMap<String, String>> requestEntity = new HttpEntity<MultiValueMap<String, String>>(params2, headers);


            ResponseEntity<BBSResponse> loginResult = restTemplate.exchange(Constants.BBS_LOGIN_URL, HttpMethod.POST, requestEntity, BBSResponse.class, params2);
            userToken = loginResult.getBody().getData();

        }
    }

    private static void publishPost(FetchHistory fetchHistory, Parser.HotCommentResult hotCommentResult, String originUrl) {

        String title = fetchHistory.getTitle();
        String thumbUrl = fetchHistory.getThumbUrl();
        String content = fetchHistory.getContent();
        Date publishTime = fetchHistory.getInitTime();
        if (content == null || content.trim().length() == 0) {
            return;
        }

        if (publishTime == null) {
            return;
        }

        Date now = new Date();
        long elapseTime = now.getTime() - publishTime.getTime();
        if (elapseTime > 1000 * 60 * 60 *36) {
            return;
        }


        updateBBSUserToken();

        String publishTimeString = new SimpleDateFormat(Constants.DATETIME_FORMAT).format(publishTime);




        RestTemplate restTemplate = new RestTemplate();


        HttpHeaders headers = new HttpHeaders();
//  //请勿轻易改变此提交方式，大部分的情况下，提交方式都是表单提交
        headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
//  //封装参数，千万不要替换为Map与HashMap，否则参数无法传递
        MultiValueMap<String, Object> params2= new LinkedMultiValueMap<String, Object>();

//  //也支持中文
        params2.add("title", title);
        params2.add("content", content);
        params2.add("token", userToken);
        params2.add("initTimeString", publishTimeString);
        params2.add("crawlerReplyRankCount", Double.toString(hotCommentResult.crawlerReplyRankCount));
        params2.add("originUrl", originUrl);


        String labelId = "1";
        String label = getTextCategory(title);
        if (label != null && label.trim().length() > 0) {
            String querylabelId = LabelClient.getRandomLabel();
            if (querylabelId != null && querylabelId.trim().length() > 0) {
                labelId = querylabelId;
            }
        }

        params2.add("labelId", labelId);
        params2.add("thumbUrl", thumbUrl);
        params2.add("crawlerReplyCount", "" + hotCommentResult.commentCount);

        HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<MultiValueMap<String, Object>>(params2, headers);


        ResponseEntity<BBSResponse> result = restTemplate.exchange(Constants.BBS_POST_URL, HttpMethod.POST, requestEntity, BBSResponse.class, params2);
        System.out.println("publish result: " + result.getStatusCode());
        if (result != null && result.getBody() != null && result.getBody().getStatus() == 200) {
            fetchHistory.setGood(true);
            App.fetchHistoryDao.saveAndFlush(fetchHistory);

            if (hotCommentResult.comments != null) {
                for (Parser.HotComment hotComment : hotCommentResult.comments) {


                    headers = new HttpHeaders();
//  //请勿轻易改变此提交方式，大部分的情况下，提交方式都是表单提交
                    headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
//  //封装参数，千万不要替换为Map与HashMap，否则参数无法传递
                    MultiValueMap<String, Object> commentParams = new LinkedMultiValueMap<String, Object>();

//  //也支持中文
                    commentParams.add("content", hotComment.content);
                    commentParams.add("token", userToken);
                    commentParams.add("postsId", result.getBody().getData());

                    HttpEntity<MultiValueMap<String, Object>> commentEntity = new HttpEntity<MultiValueMap<String, Object>>(commentParams, headers);


                    ResponseEntity<BBSResponse> commentResult = restTemplate.exchange(Constants.BBS_REPLY_URL, HttpMethod.POST, commentEntity, BBSResponse.class, commentParams);
                    System.out.println("publish comment result: " + commentResult.getStatusCode());
                }
            }
        }
    }

    /**
     * 如果需要读取已经训练好的模型，再用其进行分类，可以按照本函数的代码调用分类器
     *
     */
    public static String getTextCategory(String text) {
        // 新建分类器对象
        BasicTextClassifier classifier = new BasicTextClassifier();

        // 设置分类种类，并读取模型
        classifier.loadCategoryListFromFile("/Users/litao/Work/dev/Java/TopicRotate/code/JavaQuarkBBS/MyCrawler/novel_model/category");
        classifier.setTextClassifier(new LinearBigramChineseTextClassifier(classifier.getCategorySize()));
        classifier.getTextClassifier().loadModel("/Users/litao/Work/dev/Java/TopicRotate/code/JavaQuarkBBS/MyCrawler/novel_model");

		/*
		 * 上面三行代码等价于设置如下参数，然后初始化并运行：
		 *
		   String defaultArguments = ""
		 +  "-l  model"  // 设置您的训练好的模型的路径，这里的路径只是给出样例
		 ;
		 classifier.Init(defaultArguments.split(" "));
		 classifier.runAsLinearBigramChineseTextClassifier();
		 *
		 */

        // 之后就可以使用分类器进行分类
        int topN = 3;  // 保留最有可能的3个结果
        ClassifyResult[] result = classifier.classifyText(text, topN);
        for (int i = 0; i < topN; ++i) {
            // 输出分类编号，分类名称，以及概率值。
            System.out.println(result[i].label + "\t" +
                    classifier.getCategoryName(result[i].label) + "\t" +
                    result[i].prob);
        }
        if (result != null && result.length > 0) {
            return classifier.getCategoryName(result[0].label) ;
        }
        return "";
    }
}